{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.2"
    },
    "colab": {
      "name": "Albert.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0bdd9f4fc1b84077ab760e4ed6366680": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0dab8d958a3a4d1486058d20a700933b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dc4d344dcc2947019e5c6b5aed779db1",
              "IPY_MODEL_a23c5f4cf32c48a7ad719e609a806b6f"
            ]
          }
        },
        "0dab8d958a3a4d1486058d20a700933b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc4d344dcc2947019e5c6b5aed779db1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_0e51e8285794444f9c840440c8b14d2d",
            "_dom_classes": [],
            "description": "models/ProtAlbert/pytorch_model.bin: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 897396780,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 897396780,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3af0d5e3def54be483b020eb548a5f19"
          }
        },
        "a23c5f4cf32c48a7ad719e609a806b6f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_95abdb4f5cfb4da2866312a56a6ba283",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 856M/856M [00:20&lt;00:00, 43.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_62faa6ba081b4d4a91596be0a0f65b43"
          }
        },
        "0e51e8285794444f9c840440c8b14d2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3af0d5e3def54be483b020eb548a5f19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95abdb4f5cfb4da2866312a56a6ba283": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "62faa6ba081b4d4a91596be0a0f65b43": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca450a3ae0a840f182f7124bd2ed9fb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_5521f65691874bce89d20128f096e2a9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_384dc1ab2f6946f89e4d37865036bb37",
              "IPY_MODEL_dcc9db85774a4703b298ca8e1777af74"
            ]
          }
        },
        "5521f65691874bce89d20128f096e2a9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "384dc1ab2f6946f89e4d37865036bb37": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_faa01cf20d024a50a7187c46c1442df3",
            "_dom_classes": [],
            "description": "models/ProtAlbert/config.json: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 505,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 505,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1564d67517d64979af0eee1a57aeec9b"
          }
        },
        "dcc9db85774a4703b298ca8e1777af74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e777e7f268654a80bc0118fc6e7922f0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 505/505 [00:00&lt;00:00, 14.7kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9ba838b4fa78472da6525d37c6b3cb94"
          }
        },
        "faa01cf20d024a50a7187c46c1442df3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1564d67517d64979af0eee1a57aeec9b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e777e7f268654a80bc0118fc6e7922f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9ba838b4fa78472da6525d37c6b3cb94": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9e82580c64674b36b7af3b721f16d3cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f8656df7904346679db0c41e6ea621af",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_d9c670e425c742d19d56a7612d2a2608",
              "IPY_MODEL_774e539241bf423e8facdde802909b81"
            ]
          }
        },
        "f8656df7904346679db0c41e6ea621af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d9c670e425c742d19d56a7612d2a2608": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_34fdb5ccb1a24b6c9728f477b42c4f78",
            "_dom_classes": [],
            "description": "models/ProtAlbert/spm_model.model: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 238187,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 238187,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a1922a172b744a5489f2a2a9af529b59"
          }
        },
        "774e539241bf423e8facdde802909b81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_de929c9e848143b3a09bb428d9b6250f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 233k/233k [00:05&lt;00:00, 46.9kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2047b797bd404629bc15498df2aa7f02"
          }
        },
        "34fdb5ccb1a24b6c9728f477b42c4f78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a1922a172b744a5489f2a2a9af529b59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "de929c9e848143b3a09bb428d9b6250f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2047b797bd404629bc15498df2aa7f02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/agemagician/Prot-Transformers/blob/master/Benchmark/Albert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L9dM7fo7F2mr",
        "colab_type": "text"
      },
      "source": [
        "<h3> Benchmark ProtAlbert Model using GPU or CPU <h3>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AGYDN-r8F2mt",
        "colab_type": "text"
      },
      "source": [
        "<b>1. Load necessry libraries including huggingface transformers<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wmaiIwExGElJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "acaadd09-6362-4190-f493-97473f13fa47"
      },
      "source": [
        "!pip install -q transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 675kB 2.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.8MB 13.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 44.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 43.1MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JCiOxMzCF2mt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from transformers import AlbertModel\n",
        "import time\n",
        "from datetime import timedelta\n",
        "import os\n",
        "import requests\n",
        "from tqdm.auto import tqdm"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IM3JYihEF2mx",
        "colab_type": "text"
      },
      "source": [
        "<b>2. Set the url location of ProtAlbert and the vocabulary file<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3HE7ASh3F2mx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "modelUrl = 'https://www.dropbox.com/s/gtajtmege43ec7k/pytorch_model.bin?dl=1'\n",
        "configUrl = 'https://www.dropbox.com/s/me7zsqrnpiz043v/config.json?dl=1'\n",
        "tokenizerUrl = 'https://www.dropbox.com/s/60mg00r361vth4t/albert_vocab_model.model?dl=1'"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d6MIRgV7Gfty",
        "colab_type": "text"
      },
      "source": [
        "<b>3. Download ProtAlbert models and vocabulary files</b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ro1fmQAIGfAN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "downloadFolderPath = 'models/ProtAlbert/'"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E0-a_K4dGlqp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "modelFolderPath = downloadFolderPath\n",
        "\n",
        "modelFilePath = os.path.join(modelFolderPath, 'pytorch_model.bin')\n",
        "\n",
        "configFilePath = os.path.join(modelFolderPath, 'config.json')\n",
        "\n",
        "tokenizerFilePath = os.path.join(modelFolderPath, 'spm_model.model')"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4kdFQMV7Glpo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if not os.path.exists(modelFolderPath):\n",
        "    os.makedirs(modelFolderPath)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rEyhK1KiGllA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def download_file(url, filename):\n",
        "  response = requests.get(url, stream=True)\n",
        "  with tqdm.wrapattr(open(filename, \"wb\"), \"write\", miniters=1,\n",
        "                    total=int(response.headers.get('content-length', 0)),\n",
        "                    desc=filename) as fout:\n",
        "      for chunk in response.iter_content(chunk_size=4096):\n",
        "          fout.write(chunk)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3JcoMshhGlhQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "0bdd9f4fc1b84077ab760e4ed6366680",
            "0dab8d958a3a4d1486058d20a700933b",
            "dc4d344dcc2947019e5c6b5aed779db1",
            "a23c5f4cf32c48a7ad719e609a806b6f",
            "0e51e8285794444f9c840440c8b14d2d",
            "3af0d5e3def54be483b020eb548a5f19",
            "95abdb4f5cfb4da2866312a56a6ba283",
            "62faa6ba081b4d4a91596be0a0f65b43",
            "ca450a3ae0a840f182f7124bd2ed9fb0",
            "5521f65691874bce89d20128f096e2a9",
            "384dc1ab2f6946f89e4d37865036bb37",
            "dcc9db85774a4703b298ca8e1777af74",
            "faa01cf20d024a50a7187c46c1442df3",
            "1564d67517d64979af0eee1a57aeec9b",
            "e777e7f268654a80bc0118fc6e7922f0",
            "9ba838b4fa78472da6525d37c6b3cb94",
            "9e82580c64674b36b7af3b721f16d3cf",
            "f8656df7904346679db0c41e6ea621af",
            "d9c670e425c742d19d56a7612d2a2608",
            "774e539241bf423e8facdde802909b81",
            "34fdb5ccb1a24b6c9728f477b42c4f78",
            "a1922a172b744a5489f2a2a9af529b59",
            "de929c9e848143b3a09bb428d9b6250f",
            "2047b797bd404629bc15498df2aa7f02"
          ]
        },
        "outputId": "a0abccb9-9bb5-4869-adbe-297e25d73a57"
      },
      "source": [
        "if not os.path.exists(modelFilePath):\n",
        "    download_file(modelUrl, modelFilePath)\n",
        "\n",
        "if not os.path.exists(configFilePath):\n",
        "    download_file(configUrl, configFilePath)\n",
        "\n",
        "if not os.path.exists(tokenizerFilePath):\n",
        "    download_file(tokenizerUrl, tokenizerFilePath)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0bdd9f4fc1b84077ab760e4ed6366680",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/ProtAlbert/pytorch_model.bin', max=897396780.0, st…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ca450a3ae0a840f182f7124bd2ed9fb0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/ProtAlbert/config.json', max=505.0, style=Progress…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9e82580c64674b36b7af3b721f16d3cf",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='models/ProtAlbert/spm_model.model', max=238187.0, style=P…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dk_If-78F2m0",
        "colab_type": "text"
      },
      "source": [
        "<b>3. Load ProtAlbert Model<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Y_cWt97F2m0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = AlbertModel.from_pretrained(modelFolderPath)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "thNzFW8oF2m3",
        "colab_type": "text"
      },
      "source": [
        "<b>4. Load the model into the GPU if avilabile and switch to inference mode<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kkSNHANjF2m3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I-rqBAu8F2m6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = model.to(device)\n",
        "model = model.eval()"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ncp20TTEF2m9",
        "colab_type": "text"
      },
      "source": [
        "<b>5. Benchmark Configuration<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AJrdthVSF2m9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "min_batch_size = 8\n",
        "max_batch_size = 32\n",
        "inc_batch_size = 8\n",
        "\n",
        "min_sequence_length = 64\n",
        "max_sequence_length = 512\n",
        "inc_sequence_length = 64\n",
        "\n",
        "iterations = 10"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bNn6Y0GYF2nA",
        "colab_type": "text"
      },
      "source": [
        "<b>6. Start Benchmarking<b>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kh2DnYJ6F2nB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 748
        },
        "outputId": "b4d3cf89-97f4-4fb5-b77a-3fcf3bbb6778"
      },
      "source": [
        "device_name = torch.cuda.get_device_name(device.index) if device.type == 'cuda' else 'CPU'\n",
        "\n",
        "with torch.no_grad():\n",
        "    print((' Benchmarking using ' + device_name + ' ').center(80, '*'))\n",
        "    print(' Start '.center(80, '*'))\n",
        "    for sequence_length in range(min_sequence_length,max_sequence_length+1,inc_sequence_length):\n",
        "        for batch_size in range(min_batch_size,max_batch_size+1,inc_batch_size):\n",
        "            start = time.time()\n",
        "            for i in range(iterations):\n",
        "                input_ids = torch.randint(1, 20, (batch_size,sequence_length)).to(device)\n",
        "                results = model(input_ids)[0].cpu().numpy()\n",
        "            end = time.time()\n",
        "            ms_per_protein = (end-start)/(iterations*batch_size)\n",
        "            print('Sequence Length: %4d \\t Batch Size: %4d \\t Ms per protein %4.2f' %(sequence_length,batch_size,ms_per_protein))\n",
        "        print(' Done '.center(80, '*'))\n",
        "    print(' Finished '.center(80, '*'))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "******************* Benchmarking using Tesla P100-PCIE-16GB ********************\n",
            "************************************ Start *************************************\n",
            "Sequence Length:   64 \t Batch Size:    8 \t Ms per protein 0.04\n",
            "Sequence Length:   64 \t Batch Size:   16 \t Ms per protein 0.04\n",
            "Sequence Length:   64 \t Batch Size:   24 \t Ms per protein 0.04\n",
            "Sequence Length:   64 \t Batch Size:   32 \t Ms per protein 0.04\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  128 \t Batch Size:    8 \t Ms per protein 0.09\n",
            "Sequence Length:  128 \t Batch Size:   16 \t Ms per protein 0.08\n",
            "Sequence Length:  128 \t Batch Size:   24 \t Ms per protein 0.08\n",
            "Sequence Length:  128 \t Batch Size:   32 \t Ms per protein 0.08\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  192 \t Batch Size:    8 \t Ms per protein 0.13\n",
            "Sequence Length:  192 \t Batch Size:   16 \t Ms per protein 0.12\n",
            "Sequence Length:  192 \t Batch Size:   24 \t Ms per protein 0.12\n",
            "Sequence Length:  192 \t Batch Size:   32 \t Ms per protein 0.12\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  256 \t Batch Size:    8 \t Ms per protein 0.16\n",
            "Sequence Length:  256 \t Batch Size:   16 \t Ms per protein 0.16\n",
            "Sequence Length:  256 \t Batch Size:   24 \t Ms per protein 0.16\n",
            "Sequence Length:  256 \t Batch Size:   32 \t Ms per protein 0.16\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  320 \t Batch Size:    8 \t Ms per protein 0.20\n",
            "Sequence Length:  320 \t Batch Size:   16 \t Ms per protein 0.20\n",
            "Sequence Length:  320 \t Batch Size:   24 \t Ms per protein 0.20\n",
            "Sequence Length:  320 \t Batch Size:   32 \t Ms per protein 0.19\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  384 \t Batch Size:    8 \t Ms per protein 0.24\n",
            "Sequence Length:  384 \t Batch Size:   16 \t Ms per protein 0.24\n",
            "Sequence Length:  384 \t Batch Size:   24 \t Ms per protein 0.24\n",
            "Sequence Length:  384 \t Batch Size:   32 \t Ms per protein 0.24\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  448 \t Batch Size:    8 \t Ms per protein 0.28\n",
            "Sequence Length:  448 \t Batch Size:   16 \t Ms per protein 0.28\n",
            "Sequence Length:  448 \t Batch Size:   24 \t Ms per protein 0.28\n",
            "Sequence Length:  448 \t Batch Size:   32 \t Ms per protein 0.28\n",
            "************************************* Done *************************************\n",
            "Sequence Length:  512 \t Batch Size:    8 \t Ms per protein 0.33\n",
            "Sequence Length:  512 \t Batch Size:   16 \t Ms per protein 0.32\n",
            "Sequence Length:  512 \t Batch Size:   24 \t Ms per protein 0.32\n",
            "Sequence Length:  512 \t Batch Size:   32 \t Ms per protein 0.32\n",
            "************************************* Done *************************************\n",
            "*********************************** Finished ***********************************\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}